package wikiextract.nlp.trainingset.x;

import static wikiextract.nlp.trainingset.x.Settings.NUM_CROSS_LEXICONS;
import static wikiextract.nlp.trainingset.x.RunSecondPass.secondFeatureIds;
import static wikiextract.nlp.trainingset.x.RunSecondPass.featureIds;
import static wikiextract.nlp.trainingset.x.RunSecondPass.getSecondFeatureId;

import java.util.ArrayList;
import java.util.List;

import wikiextract.data.util.DynamicDoubleArray;

public class SecondPassNumdists {

	static int NUM_GAUSSIANS = 4;
	
	static class NumdistFeature {
		String featureName;
		int featureId;
		Numdists numdists;
		NumdistFeature(String featureName) {
			this.featureName = featureName;
			this.featureId = getSecondFeatureId(featureName, secondFeatureIds, featureIds);
			this.numdists = new Numdists();
		}
	}
	
	static class Numdists {
		double expansionFactor = 1;
		Numdist train;
		List<Numdist> chunkTrain;
		List<Numdist> chunkTest;
		
		Numdists() {
			train = new Numdist();
			//test = new Lexicon();
			chunkTrain = new ArrayList<Numdist>();
			chunkTest = new ArrayList<Numdist>();
			for (int c = 0; c < NUM_CROSS_LEXICONS; c++) {
				chunkTrain.add(new Numdist());
				chunkTest.add(new Numdist());
			}
		}
	}
	
	static class Numdist {
		Gaussian g = new Gaussian();
		double[] index;
	}
	
	static void gaussians(List<List<FeaturesRecord>> train,
			List<NumdistFeature> numdists, String featureName, int label, int word) {
		for (int i=0; i < NUM_GAUSSIANS; i++) {
			double ef = Math.pow(2.0, i/*+1*/);
			//double ef = (i+1);
			
			NumdistFeature nf = new NumdistFeature(featureName + "_" + i);
			Numdists ndi = nf.numdists;
			ndi.expansionFactor = ef;
	
			for (List<FeaturesRecord> articleExamples : train)
				add(articleExamples, ndi.train.g, label, word);
	
			// break up trainingExamples into max NUM_CROSS_LISTS chunks
			// assign articles to chunks
			for (int k = 0; k < train.size(); k++) {
				int c = k % NUM_CROSS_LEXICONS;
	
				// add seeds to all other chunks, current one only for testing
				for (int d = 0; d < Math.min(NUM_CROSS_LEXICONS, train.size()); d++) {					 
					if (c != d) {
						add(train.get(k), ndi.chunkTrain.get(d).g, label, word);
					}
				}
			}
			numdists.add(nf);
		}
	}

	// word: 0 = all, 1 = only first word of segment
	private static void add(List<FeaturesRecord> article, Gaussian g,
			int label, int word) {
		for (FeaturesRecord ex : article) {
			for (int m = 0; m < ex.matchPos.length; m++) {
				if (ex.matchLabels[m] != label) continue;
				byte[] t = ex.matchPos[m];
				StringBuilder sb = new StringBuilder();
				for (int j = t[0]; j < t[1]; j++) {					
					if (j > t[0])
						sb.append(" ");
					sb.append(ex.tokens[j].toLowerCase());
					if (word == 1) break; // only first word
				}
				try {
					double v = Double.parseDouble(sb.toString());
					g.add(v);
				} catch (NumberFormatException e) { }
			}
		}	
	}
	
	static void createNumdistIndexes(List<NumdistFeature> numdists) {
		for (NumdistFeature nf : numdists) {
			Numdists ns = nf.numdists;
			
			createIndex(ns.train, ns.expansionFactor);
			for (Numdist nd : ns.chunkTest)
				createIndex(nd, ns.expansionFactor);
			for (Numdist nd : ns.chunkTrain)
				createIndex(nd, ns.expansionFactor);
		}
	}
	
	private static void createIndex(Numdist nd, double expansionFactor) {
		Gaussian g = nd.g;
		nd.index = new double[] {
			g.mean() - expansionFactor*g.stddev(),
			g.mean() + expansionFactor*g.stddev()};		
	}

	static void numdistFeatures(FeaturesRecord o, boolean test, double[] interval,
			List<Integer>[] fts, int featureId) {
		for (int i = 0; i < o.tokens.length; i++) {
			String tok = o.tokens[i].toLowerCase();

			double d = 0;
			try {
				d = Double.parseDouble(tok);
			} catch (Exception ex) { continue; }
			
			if (d >= interval[0] && d <= interval[1])
				fts[i].add(featureId);
		}
	}

	static class Gaussian {
		DynamicDoubleArray values = new DynamicDoubleArray();
		
		void add(double v) {
			values.append(v);
			dirty = true;
			total++;
		}
		
		private int total = 0;
		
		private boolean dirty = true;
		private double mean = 0;
		private double variance = 0;

		public void reset() {
			values.clear();
			total = 0;
			dirty = true;
		}
		
		public double stddev() {
			if (dirty) refreshModel();
			return Math.sqrt(variance);
		}
		
		public double mean() {
			if (dirty) refreshModel();
			return mean;
		}
		
		public double predict(double v) {
			if (dirty) refreshModel();
			double pdf = 1.0 / (mean* Math.sqrt(2*Math.PI)) * Math.exp(- (v - mean)*(v - mean) / (2*variance));
			return pdf;
		}
		
		private void refreshModel() {
			// compute mean
			double[] d = values.getData();
			double sum = 0;
			for (int i=0; i < values.length(); i++)
				sum += d[i];
			mean = sum / values.length();
			
			// compute variance
			sum = 0;
			for (int i=0; i < values.length(); i++)
				sum += (d[i] - mean)*(d[i] - mean);
			variance = sum / values.length();
			dirty = false;
		}
	}
}

